<?php
/**
 * Created by JetBrains PhpStorm.
 * User: xiaobai
 * Date: 13-6-27
 * Time: 上午9:58
 * To change this template use File | Settings | File Templates.
 */

class Crawl_abstract {

    const PRROXY_IP = 'proxy';
    const MOBILITY_IP = 'mobility';
    const IP_USABILITY_TEST_TIMEOUT = 1;
    const LOG_LEVEL_ERROR = 'error';
    const LOG_LEVEL_WARING = 'waring';
    const LOG_LEVEL_NOTICE = 'notice';
    const DEFAULT_PROXY_PORT = 80;
    const METHOD_POST = 'POST';
    const METHOD_GET = 'GET';
    const LOG_START_TAG = 'START------------------------------------------------------------------------------------------------------------------------------------START';
    public $_cookie_error_code = array(82440414, 82440413);
    public $_illegal_error = array(82440418);
    public $_error_string = 'error';
    public $_null_data_array_key = 'data';
    public $_ip_usability_test_controller = 'ip/usability';
    public $_log_dir = 'keyword';
    public $_result_log_dir = 'result';
    public $_null_tab;
    public $_crawl_from = array(
        'sina_html' => 'Sina Html',
        'sina_wap'  => 'Sina Wap',
    	'sina_phone'  => 'Sina Phone',
        'yunyun_wap'=> 'YunYun WAP',
        'yunyun_html'=> 'YunYun Html',
        'netease_html' => 'Netease Html',
        'sohu_api'     => 'SOHU API',
        'tencent_api'  => 'TENCENT API',
        'tencent_html'  => 'TENCENT HTML',
        'tencent_client'  => 'TENCENT CLIENT',
        'user_check'    => 'Other Web Site'
    );
    public  $_proxy_ip = array();
    public  $_use_proxy;

    public  $_platform;

    function __construct()
    {
        $this->_ci = &get_instance();
        $this->_ci->load->helper('url');
        $this->_ci->load->library('error_code_library');
    }


    /**抓取入口
     * @param array $options CURL 参数
     * @param bool $use_proxy 是否使用代理 Default FALSE 不适用代理 直接抓取
     * @param string $proxy_from 代理IP来源 肉鸡IP / ADSL 动态拨号IP
     * @return array('result' => 抓取内容, 'proxy_ip' => 代理IP)
     */

    public function get_content($parameters = array(), $proxy_from = FALSE) {
        $this->_platform = is_null($this->_platform) ? $this->_ci->input->post('platform') : $this->_platform;

        if($proxy_from == FALSE) {
            return $this->execute_crawl($parameters, $proxy_from);
        } else {
            //$get_basic_parameters = $this->get_basic_parameters($proxy_from);
            //crawl_log
            $get_ip_result = $this->get_ip($proxy_from);

            if(array_key_exists($this->_error_string, $get_ip_result)) {
                //如果获取失败或验证可用性失败 直接返回error数组
                //crawl_log
                //$get_basic_parameters['ip_error'] = json_encode($get_ip_result);
                //$this->create_crawl_log($get_basic_parameters);

                return $get_ip_result;
            } else {
                //crawl_log
                //$get_basic_parameters['ip_error'] = NULL;
                $get_basic_parameters['use_ip'] = $get_ip_result['ip_string'];

                $parameters['proxy']['ip'] = $get_ip_result['ip_string'];
                $use_ip = $get_ip_result['ip_array'];
            }

			//修复腾讯平台 clientip参数
            if(array_key_exists('postData', $parameters) && array_key_exists('clientip', $parameters['postData'])) {
                $parameters['postData']['clientip'] = $use_ip['ip'];
            }

            $result = $this->execute_crawl($parameters, $proxy_from, $get_ip_result);

            //crawl_log
            /*if(is_array($result) && array_key_exists($this->_error_string, $result)) {
                $get_basic_parameters['curl_result'] = 2;
                $get_basic_parameters['curl_return'] = json_encode($result);

            } else {
                $get_basic_parameters['curl_result'] = 1;
                $get_basic_parameters['curl_return'] = $result;
            }
            $get_basic_parameters['end_time'] = time();
            $this->create_crawl_log($get_basic_parameters);*/

            //先不记录抓取的次数
            $this->update_use_ip($use_ip['id'], $proxy_from);
            
            if(is_array($result) && array_key_exists($this->_error_string, $result)) {
                return $result;
            }
            return array('result' => $result, 'use_ip' => $use_ip);
        }
    }

    public function crawl_use_proxy_ip(){

    }

    public function retry_crawl(){

    }

    /**
     * 获取抓取将使用的IP 并测试是否可用
     * @param string $proxy_from 代理|动态ADSL
     * @return array
     * 验证通过 return  array('ip_string' => $ip_string, 'ip_array' => $ip_array)
     * 验证不通过 return array('error' => string, 'code' => error_code ,'msg' => error message);
     */
    public function get_ip($proxy_from = self::PRROXY_IP) {
        if($proxy_from == self::PRROXY_IP) {
            //获取代理IP
            $ip = $this->make_proxy_ip();
            if(array_key_exists($this->_error_string, $ip)) {
                return $ip;
            } else {
                //验证代理IP
                $ip_usability_test_result = $this->ip_usability_test($ip);
                if($ip_usability_test_result !== TRUE) {
                    //验证失败直接返回验证错误信息
                    return $ip_usability_test_result;
                } else {
                    $this->update_proxy_ip_use_vtime($ip['ip_array']['id']);
                }
            }
        } else {
            $ip = $this->make_mobility_ip();
            /// 流动性
        }
        return $ip;
    }

    /**
     * 获取代理IP地址
     * @return array
     * 正确 return  array('ip_string' => $ip_string, 'ip_array' => $ip_array)
     * 错误 return array('error' => string, 'code' => error_code ,'msg' => error message);
     */
    public function make_proxy_ip() {
        $proxy_ip = $this->get_proxy_ip();
        log_message(self::LOG_LEVEL_NOTICE, 'Will Get Proxy Ip: '.json_encode($proxy_ip), FALSE, $this->_log_dir);
        if(array_key_exists($this->_error_string, $proxy_ip)) {
            log_message(self::LOG_LEVEL_WARING, json_encode($proxy_ip), FALSE, $this->_log_dir);
            return $proxy_ip;
        } else {
            $proxy_ip_result = $this->make_ip_result($proxy_ip, self::PRROXY_IP);

            return $proxy_ip_result;
        }
    }

    /**
     * 获取动态ADSL IP地址
     * @return array
     * 正确 return  array('ip_string' => $ip_string, 'ip_array' => $ip_array)
     * 错误 return array('error' => string, 'code' => error_code ,'msg' => error message);
     */
    public function make_mobility_ip() {
        $mobility_ip = $this->get_mobility_ip();
        log_message(self::LOG_LEVEL_NOTICE, 'Will Get ADSL Ip: '.json_encode($mobility_ip), FALSE, $this->_log_dir);
        if(array_key_exists($this->_error_string, $mobility_ip) === TRUE) {
            log_message(self::LOG_LEVEL_WARING, json_encode($mobility_ip), FALSE, $this->_log_dir);
            return $mobility_ip;
        } else {
            $mobility_ip_result = $this->make_ip_result($mobility_ip, self::MOBILITY_IP);
            return $mobility_ip_result;
        }

    }

    /**
     * 拼装获取IP结果
     * @param $ip_array
     * @param string $proxy_from
     * @return  array('ip_string' => $ip_string, 'ip_array' => $ip_array)
     */
    public function make_ip_result($ip_array, $proxy_from = self::PRROXY_IP) {
        if($proxy_from == self::PRROXY_IP) {
            if(isset($ip_array['port']) || $ip_array['port'] > 0) {
                $ip_string = $ip_array['ip'].':'.$ip_array['port'];
            } else {
                $ip_string  = $ip_array['ip'].self::DEFAULT_PROXY_PORT;
            }
            log_message(self::LOG_LEVEL_NOTICE, 'Will USE Proxy Ip: '.$ip_string , FALSE, $this->_log_dir);
        } else {
            $ip_string = $ip_array['ip'].':'.$ip_array['port'];
            log_message(self::LOG_LEVEL_NOTICE, 'Will USE ADSL Ip: '.$ip_string, FALSE, $this->_log_dir);
        }
        return array('ip_string' => $ip_string, 'ip_array' => $ip_array);
    }

    /**
     * 验证代理IP可用性
     * @param $ip array('ip_string' => $ip_string, 'ip_array' => $ip_array)
     * @param string $proxy_from  代理|动态ADSL
     * @return TRUE | Array
     * 正确 TRUE
     * 错误 return array('error' => string, 'code' => error_code ,'msg' => error message);
     */
    public function ip_usability_test($ip, $proxy_from = self::PRROXY_IP) {
        if(is_array($ip) && array_key_exists($this->_error_string, $ip)) {
            return $ip;
        } else {
            //拼装参数
            $parameters['url'] = site_url($this->_ip_usability_test_controller);
            $parameters['proxy']['ip'] = $ip['ip_string'];
            $parameters['time_out'] = self::IP_USABILITY_TEST_TIMEOUT;
            //发送curl请求
            $crawl_ip_usability_test = $this->execute_crawl($parameters, $proxy_from, $ip);

            if(is_array($crawl_ip_usability_test) && array_key_exists($this->_error_string, $crawl_ip_usability_test)) {
                //CRUL错误
                log_message(self::LOG_LEVEL_WARING, 'Ip Usability Test Failed: '.json_encode($crawl_ip_usability_test), FALSE, $this->_log_dir);
                return $crawl_ip_usability_test;
            } else {
                $crawl_ip_usability_test_result = json_decode($crawl_ip_usability_test, TRUE);
                if(is_null($crawl_ip_usability_test_result) || $crawl_ip_usability_test_result['result'] != TRUE) {
                    //获取结果不完整
                    $this->label_proxy_table($ip['ip_array'], $proxy_from);
                    log_message(self::LOG_LEVEL_WARING, 'Ip Usability Test Failed: Crawl Result Error '.$crawl_ip_usability_test, FALSE, $this->_log_dir);
                    return $this->_ci->error_code_library->getError('curl', 'runtime', 'ip_usability_failed', 'Use '.$ip['ip_string'].' Crawl Result:'.$crawl_ip_usability_test);
                } else {
                    log_message(self::LOG_LEVEL_NOTICE, 'Ip Usability Test OK: '.$ip['ip_string'], FALSE, $this->_log_dir);
                    return TRUE;
                }
            }
        }
    }

    /**
     * 更新IP使用次数
     * @param $ip_id
     * @param string $proxy_from
     */
    public function update_use_ip($ip_id, $proxy_from = self::PRROXY_IP) {
        if($proxy_from == self::PRROXY_IP) {
            $this->update_proxy_ip_use_count($ip_id);
        } else {
            $this->update_mobility_ip_use_count($ip_id);
        }
    }
    /**执行抓取
     * @param array $parameters CURL参数
     * @return 抓取内容
     */
    public function execute_crawl($parameters = array(), $proxy_from = self::PRROXY_IP, $use_ip = array()) {
        $curl = $this->make_curl_parameter($parameters, $proxy_from);
        if($curl == FALSE) {
            log_message(self::LOG_LEVEL_WARING, 'CURL Error: Can`t Find The Mast Parameter', FALSE, $this->_log_dir);
            return $this->_ci->error_code_library->getError('curl', 'params', 'no_must_parameter', "In ".__FUNCTION__." Use Parameters ".json_encode($parameters));
        }

        $result = curl_exec($curl);

		//临时添加,调试抓取返回结果 add by hejigui
        if($this->_ci->input->post('soften_admin_debug')){
        	//var_dump($proxy_from);
        	//var_dump($parameters);
        	
        	//file_put_contents("./soften_admin_debug.log", $this->return_utf8_code($result));
        	//var_dump($this->return_utf8_code($result));
        }

        if($result == FALSE) {
            $error_string = curl_error($curl);
            $http_info = curl_getinfo($curl);
            if(isset($parameters['proxy'])) {
                $error_string = 'Use '.$parameters['proxy']['ip'].' '.$error_string;
                //使用CURL 使用ADSL请求 返回超时 无法连接等CURL错误时 不再标记IP不可用
                if($http_info['http_code'] != 200){
                	$this->label_proxy_table($use_ip['ip_array'], $proxy_from);
                }
                if($this->_platform == 'sina') {
                    $this->update_mobility_ip_error_count($use_ip['ip_array']['id'],$error_string);
                }
            }
            log_message(self::LOG_LEVEL_WARING, 'CURL Error: '.curl_error($curl), FALSE, $this->_log_dir);
            $result = $this->_ci->error_code_library->getError('curl', 'runtime', 'curl_post',$error_string);
        } else {
            log_message(self::LOG_LEVEL_NOTICE, 'CURL Return Content: OK', FALSE, $this->_log_dir);
        }
        curl_close($curl);
        if(is_string($result)) {
            $result = trim($result);
        }
        return $result;
    }

    /**
     * @param array $parameters CURL 参数
     * @return CURL 对象
     */
    public function make_curl_parameter($parameters = array(), $proxy_from = self::PRROXY_IP) {
        if(count($parameters) == 0 || !isset($parameters['url'])) {
            return false;
        }

        $ch = curl_init();

        if(isset($parameters['method']) && $parameters['method'] == self::METHOD_GET) {
            if(isset($parameters['postData'])) {
                $parameters['url'] = $parameters['url'].'?'.http_build_query($parameters['postData']);
            }
        }

        if(isset($parameters['method']) && $parameters['method'] == self::METHOD_POST) {
            curl_setopt($ch, CURLOPT_POST, 1);
            if($parameters['postData']) {
                curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($parameters['postData']));
            }
        }

        if(isset($parameters['proxy'])) {
            curl_setopt($ch, CURLOPT_PROXY, $parameters['proxy']['ip']);
        }

        if($this->_ci->config->item('use_socks5') === TRUE && $proxy_from == self::MOBILITY_IP) {
            curl_setopt($ch, CURLOPT_PROXYTYPE , CURLPROXY_SOCKS5);
        }

        if($this->_ci->config->item('adsl_proxy_user_auth') === TRUE && $proxy_from == self::MOBILITY_IP) {
            $user_name = $this->_ci->config->item('adsl_proxy_user_name');
            $user_pass = $this->_ci->config->item('adsl_proxy_user_password');
            curl_setopt($ch, CURLOPT_PROXYUSERPWD,$user_name.':'.$user_pass);
        }
        //$parameters['url'] = 'http://api.crawl.com/test/page';
        curl_setopt($ch, CURLOPT_URL, $parameters['url']);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION,1);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($ch, CURLOPT_DNS_USE_GLOBAL_CACHE, FALSE);


        if(isset($parameters['referer']) && $parameters['referer']) {
            curl_setopt($ch, CURLOPT_REFERER, $parameters['referer']);
        }else{
        	curl_setopt($ch, CURLOPT_REFERER, $parameters['url']);
        }

        if(array_key_exists('time_out', $parameters)) {
            curl_setopt($ch, CURLOPT_TIMEOUT, $parameters['time_out']);
        } else {
            curl_setopt($ch, CURLOPT_TIMEOUT, $this->_ci->config->item('curl_timeout'));
        }

        if(isset($parameters['returnHeader']) && $parameters['returnHeader']) {
            curl_setopt($ch, CURLOPT_HEADER, 1);
        }

        if(isset($parameters['returnBody']) && !$parameters['returnBody']) {
            curl_setopt($ch, CURLOPT_NOBODY, 1);
        }

        if(isset($parameters['httpHeader'])  && !empty($parameters['httpHeader'])) {
            curl_setopt($ch, CURLOPT_HTTPHEADER, $parameters['httpHeader']);
        }

        if(isset($parameters['userAgent'])  && $parameters['userAgent']) {
            curl_setopt($ch, CURLOPT_USERAGENT, $parameters['userAgent']);
        }


        if(isset($parameters['cookie']) && $parameters['cookie']) {
            if($parameters['cookie_str'] == true) {
                $cookie = $parameters['cookie'];
                curl_setopt($ch, CURLOPT_COOKIE, $cookie);
            } else {
                $file = "/tmp/proxy_".time().substr(md5($parameters['cookie']), 0, 5);
                $cookie = json_decode($parameters['cookie'], true);
                if(!is_array($cookie)) {
                    return false;
                }
                $fp = fopen($file, 'w');
                foreach($cookie as $row) {
                    fputs($fp, $row);
                }
                curl_setopt($ch, CURLOPT_COOKIEFILE, $file);
            }

            //@unlink($file);
        }

        return $ch;
    }

    /**
     * 标记IP为不可用
     * @param array $use_ip
     * @param string $use_proxy
     */
    public function label_proxy_table($use_ip = array(), $use_proxy = self::PRROXY_IP) {
        if(count($use_ip) > 0 && array_key_exists('ip', $use_ip)) {
            if($use_proxy == self::PRROXY_IP) {
                $this->label_proxy_ip($use_ip);
            } else {
                $this->label_mobility_ip($use_ip['id']);
                $this->update_mobility_ip_error_count($use_ip['id'],'need_verified_code');
            }
        }
    }


    public function change_code($crawl_result) {
        $crawl_result = preg_replace('/[^\x20-\xff]*/', "", $crawl_result); //清除不可见字符
        $crawl_result_array = json_decode($crawl_result, TRUE);
        if(is_array($crawl_result_array) ) {
            return $crawl_result_array;
        } else {
            return FALSE;
        }
    }

    public function write_log_stary_tag() {
        log_message(self::LOG_LEVEL_NOTICE, self::LOG_START_TAG, FALSE, $this->_log_dir);
    }

    public function get_proxy_ip() {
        $this->_ci->load->model('universal/Proxy_model');
        return $this->_ci->Proxy_model->get_proxy_ip();
    }

    public function get_mobility_ip() {
        $this->_ci->load->model('universal/Mobility_ip_model');
        return $this->_ci->Mobility_ip_model->get_mobility_ip($this->_platform);
    }

    public function get_cookie($need_delay = true) {
        $this->_ci->load->model('keyword/Cookie_model');
        return $this->_ci->Cookie_model->get_cookie($this->_platform, $need_delay);
    }

    public function get_unlock_cookie($platform) {
    	$this->_ci->load->model('keyword/Cookie_model');
    	return $this->_ci->Cookie_model->get_unlock_cookie($platform);
    }
    
    public function get_user_token($platform) {
        $this->_ci->load->model('universal/User_token_model');
        $where['platform'] = $platform;
        return $this->_ci->User_token_model->get_token_by_platform($where);
    }

    public function update_user_token_total($id) {
        $this->_ci->load->model('universal/User_token_model');
        return $this->_ci->User_token_model->increase_total($id);
    }

    public function update_proxy_ip_use_count($id) {
        $this->_ci->load->model('universal/Proxy_model');
        return $this->_ci->Proxy_model->update_proxy_ip_use_count($id);
    }

    public function update_proxy_ip_use_vtime($id) {
        $this->_ci->load->model('universal/Proxy_model');
        return $this->_ci->Proxy_model->update_proxy_ip_vtime($id);
    }

    public function update_mobility_ip_use_count($id){
        $this->_ci->load->model('universal/Mobility_ip_model');
        return $this->_ci->Mobility_ip_model->update_mobility_ip_use_count($id, $this->_platform);
    }

    public function update_mobility_ip_error_count($id, $error_string){
        $this->_ci->load->model('universal/Mobility_ip_error_model');
        return $this->_ci->Mobility_ip_error_model->update_mobility_ip_error_count($id, $error_string);
    }

    public function update_cookie_use_count($id) {
        $this->_ci->load->model('keyword/Cookie_model');
        return $this->_ci->Cookie_model->update_cookie_use_count($id);
    }

    public function update_cookie_wap_count($id) {
        $this->_ci->load->model('keyword/Cookie_model');
        return $this->_ci->Cookie_model->update_cookie_wap_count($id);
    }

    public function label_proxy_ip($ip) {
        $this->_ci->load->model('universal/Proxy_model');
        return $this->_ci->Proxy_model->label_proxy_ip($ip);
    }

    public function label_mobility_ip($id) {
        $this->_ci->load->model('universal/Mobility_ip_model');
        return $this->_ci->Mobility_ip_model->label_mobility_ip($id, $this->_platform);
    }

    public function label_cookie($id) {
        $this->_ci->load->model('keyword/Cookie_model');
        return $this->_ci->Cookie_model->label_cookie($id);
    }
    
    public function unlock_cookie($id) {
    	$this->_ci->load->model('keyword/Cookie_model');
    	return $this->_ci->Cookie_model->unlock_cookie($id);
    }

    public function find_in_cache() {
    	//若传入no_cache参数..则不再走cache...
    	if($this->_ci->input->post('no_cache')){
    		return;
    	}

        $this->_ci->load->model('keyword/Keyword_crawl_model');
        $cache_value = $this->_ci->Keyword_crawl_model->find_in_cache();
        if($cache_value !== FALSE) {
            $history_result = json_decode($cache_value, TRUE);
            return $history_result['result'];

        } else {
            $result = $this->_ci->Keyword_crawl_model->find_in_db();
            if(!is_null($result)) {
                return $result['content'];
            } else {
                return $result;
            }
        }

    }

    /**
     * 数据转码
     * @param $data
     * @return array|string
     */
    public function return_utf8_code($data) {
        if(is_array($data)) {
            foreach($data as $key => $value) {
                $data[$key] = $this->return_utf8_code($value);
            }
            return $data;
        } else {
        	//return $data;
            return $data = mb_check_encoding($data, 'UTF-8') ? $data : utf8_encode($data);
        }
    }

    public function create_crawl_log($data) {
        $this->_ci->load->model('universal/Crawl_log_model');
        $this->_ci->Crawl_log_model->create_cral_log($data);
    }

    public function not_illegal_error($error_array = array()) {
        if(array_key_exists('error_code', $error_array) && in_array($error_array['error_code'], $this->_illegal_error)) {
            return FALSE;
        } else {
            return TRUE;
        }
    }
}
